{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "3c7d5caa-80c5-407f-8c9a-48a9f08d690a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>category</th>\n",
       "      <th>description</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Biamp Rack Products</td>\n",
       "      <td>Digital Audio Processors</td>\n",
       "      <td>“High recognition value, uniform aesthetics an...</td>\n",
       "      <td>Product Name: Biamp Rack Products;\\n\\nProduct ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>V33</td>\n",
       "      <td>Video Camera</td>\n",
       "      <td>The V33 livestreaming video camera ensures hig...</td>\n",
       "      <td>Product Name: V33;\\n\\nProduct Category: Video ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HP LaserJet 5000-6000 and E700-E800 Series MFPs</td>\n",
       "      <td>Multi-Function Printers</td>\n",
       "      <td>The HP LaserJet 5000 to 6000 Series and E700 t...</td>\n",
       "      <td>Product Name: HP LaserJet 5000-6000 and E700-E...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Meaco Arete One 20L Dehumidifier</td>\n",
       "      <td>Heating and Air Conditioning Technology</td>\n",
       "      <td>The Meaco Arete One Dehumidifier is characteri...</td>\n",
       "      <td>Product Name: Meaco Arete One 20L Dehumidifier...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>théATRE Glass Container for Loose Leaf Tea</td>\n",
       "      <td>Food Containers</td>\n",
       "      <td>The design and colouring of the théATRE Glass ...</td>\n",
       "      <td>Product Name: théATRE Glass Container for Loos...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           product  \\\n",
       "0                              Biamp Rack Products   \n",
       "1                                              V33   \n",
       "2  HP LaserJet 5000-6000 and E700-E800 Series MFPs   \n",
       "3                 Meaco Arete One 20L Dehumidifier   \n",
       "4       théATRE Glass Container for Loose Leaf Tea   \n",
       "\n",
       "                                  category  \\\n",
       "0                 Digital Audio Processors   \n",
       "1                             Video Camera   \n",
       "2                  Multi-Function Printers   \n",
       "3  Heating and Air Conditioning Technology   \n",
       "4                          Food Containers   \n",
       "\n",
       "                                         description  \\\n",
       "0  “High recognition value, uniform aesthetics an...   \n",
       "1  The V33 livestreaming video camera ensures hig...   \n",
       "2  The HP LaserJet 5000 to 6000 Series and E700 t...   \n",
       "3  The Meaco Arete One Dehumidifier is characteri...   \n",
       "4  The design and colouring of the théATRE Glass ...   \n",
       "\n",
       "                                                text  \n",
       "0  Product Name: Biamp Rack Products;\\n\\nProduct ...  \n",
       "1  Product Name: V33;\\n\\nProduct Category: Video ...  \n",
       "2  Product Name: HP LaserJet 5000-6000 and E700-E...  \n",
       "3  Product Name: Meaco Arete One 20L Dehumidifier...  \n",
       "4  Product Name: théATRE Glass Container for Loos...  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from datasets import load_dataset\n",
    "from datasets import Dataset\n",
    "import os\n",
    "\n",
    "data_file_path = os.path.join(\"../data\", '21183.parquet')\n",
    "rd_ds = pd.read_parquet(data_file_path)\n",
    "rd_ds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "32e50c89-5e9e-494f-8fb8-d6ee901d85ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "columns: Index(['product', 'category', 'description', 'text'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(f'columns: {rd_ds.columns}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "3de945c6-6da2-4814-b6bf-14b4693d28f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Create a detailed description for the following product: Biamp Rack Products, belonging to category: Digital Audio Processors'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将商品名称和商品类别组合成指令\n",
    "rd_ds['instruction'] = 'Create a detailed description for the following product: ' + rd_ds[\n",
    "    'product'] + ', belonging to category: ' + rd_ds['category']\n",
    "\n",
    "rd_ds.instruction[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "0b396141-1d6d-444b-9e1d-3edc116da217",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instruction</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18952</th>\n",
       "      <td>Create a detailed description for the followin...</td>\n",
       "      <td>The CG8565 is a gaming PC offering space for h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12584</th>\n",
       "      <td>Create a detailed description for the followin...</td>\n",
       "      <td>The iSHOXS BullBar ProX mount can be used to a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5702</th>\n",
       "      <td>Create a detailed description for the followin...</td>\n",
       "      <td>The S81 Pro focuses on two things: outstanding...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20503</th>\n",
       "      <td>Create a detailed description for the followin...</td>\n",
       "      <td>The CenFlex superfinish machine is designed fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2480</th>\n",
       "      <td>Create a detailed description for the followin...</td>\n",
       "      <td>The THALION S gas absorption heat pump uses na...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             instruction  \\\n",
       "18952  Create a detailed description for the followin...   \n",
       "12584  Create a detailed description for the followin...   \n",
       "5702   Create a detailed description for the followin...   \n",
       "20503  Create a detailed description for the followin...   \n",
       "2480   Create a detailed description for the followin...   \n",
       "\n",
       "                                             description  \n",
       "18952  The CG8565 is a gaming PC offering space for h...  \n",
       "12584  The iSHOXS BullBar ProX mount can be used to a...  \n",
       "5702   The S81 Pro focuses on two things: outstanding...  \n",
       "20503  The CenFlex superfinish machine is designed fo...  \n",
       "2480   The THALION S gas absorption heat pump uses na...  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提取instruction、description两列\n",
    "rd_ds = rd_ds[['instruction', 'description']]\n",
    "# 随即采样5000条样本用于后续模型微调\n",
    "rd_df_sample = rd_ds.sample(n=5000, random_state=42)\n",
    "rd_df_sample.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "4aff251a-e357-4968-a321-e1850b9ec1e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义指令微调模板并将处理后的instruction格式化到模板中，用于后续监督学习指令微调过程\n",
    "template = \"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
    "\n",
    "### Instruction:\n",
    "\n",
    "{}\n",
    "\n",
    "### Response:\\n\"\"\"\n",
    "\n",
    "rd_df_sample['prompt'] = rd_df_sample[\"instruction\"].apply(lambda x: template.format(x))\n",
    "rd_df_sample.rename(columns={'description': 'response'}, inplace=True)\n",
    "rd_df_sample['response'] = rd_df_sample['response'] + \"\\n### End\"\n",
    "rd_df_sample = rd_df_sample[['prompt', 'response']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "7bd42db0-6f4a-4c3b-9361-e3ec79f62344",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt:\n",
      " Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
      "\n",
      "### Instruction:\n",
      "\n",
      "Create a detailed description for the following product: CG8565, belonging to category: Desktop Computer\n",
      "\n",
      "### Response:\n",
      "\n",
      "Response:\n",
      " The CG8565 is a gaming PC offering space for high-quality equipment. The Windows 7 system works with an Intel Core i7 2600K processor and supports up to 32 GB of working memory. Two graphics cards, an SSD hard drive, an efficient water-cooling system and three chassis fans guarantee excellent performance. At the push of a button, the system can be overclocked by up to  35 per cent during continuous operation.\n",
      "### End\n"
     ]
    }
   ],
   "source": [
    "for index, row in rd_df_sample.iterrows():\n",
    "    print(f'Prompt:\\n {row.prompt}')\n",
    "    print(f'Response:\\n {row.response}')\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2ee8235-6558-47f0-ae27-eccdf661fb9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "rd_ds['text'] = rd_df[\"prompt\"] + rd_df[\"response\"]\n",
    "rd_df.drop(columns=['prompt', 'response'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a49839a4-b684-4540-9afb-8dc5d4d1fca8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
